/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package ddsunase_parser;

import static ddsunase_parser.Drift_Fresh_FreshExtended.counCorrectRank;
import static ddsunase_parser.Drift_Fresh_FreshExtended.currentProjectName;

import static ddsunase_parser.Drift_Fresh_FreshExtended.maxCommentsDelay;
import static ddsunase_parser.Drift_Fresh_FreshExtended.maxDupsExp;
import static ddsunase_parser.Drift_Fresh_FreshExtended.maxFixExp;
import static ddsunase_parser.Drift_Fresh_FreshExtended.maxPPLDelay;
import static ddsunase_parser.Drift_Fresh_FreshExtended.maxSimilarity;
import static ddsunase_parser.Drift_Fresh_FreshExtended.maxTimeDelay;

import static ddsunase_parser.Drift_Fresh_FreshExtended.parseResultsFile;
import static ddsunase_parser.Drift_Fresh_FreshExtended.printValue;
import static ddsunase_parser.Drift_Fresh_FreshExtended.project;

import static ddsunase_parser.Drift_Fresh_FreshExtended.readDupsEffort;
import static ddsunase_parser.Drift_Fresh_FreshExtended.topN;
import static ddsunase_parser.Drift_Fresh_FreshExtended.treeMapDupResults;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author samy
 */
public class Drift_Fresh_FreshExtended {

    public static String type = null;
    public static String project = null;
    public static int topN = 0;
    public static double maxTimeDelay = 0;
    public static double maxCommentsDelay = 0;
    public static double maxPPLDelay = 0;
    public static double maxDupsExp = 0;
    public static double maxFixExp = 0;
    public static double maxSimilarity = 0;
    public static double[] counCorrectRank;
    public static double[] bestSwarmPosition;
    //public static List<DupResultObj> generalCompareList;
    public static TreeMap<String, List<DupResultObj>> treeMapDupResults = new TreeMap<String, List<DupResultObj>>();

    public static TreeMap<String, TreeMap<Integer, DriftObj>> mapResults = new TreeMap<String, TreeMap<Integer, DriftObj>>();

    public static TreeMap<String, TreeMap<Integer, DriftObj>> mapModels = new TreeMap<String, TreeMap<Integer, DriftObj>>();

    public static TreeMap<Integer, TreeMap<Integer, DataRange>> rangesTree;

    public static TreeMap<Integer, SortObj> sortingMap;
    public static boolean statusHistoryFiltering = false;
    public static int numberOf_Iterations;
    public static int countResolvedAtTime;
    public static int countOpenResolvedAtTime;
    public static int counMatchMissedOpen;
    public static int countMissedResolvedAtTime;
    public static String currentProjectName;

    public static Map<String, String> startHashMap = new HashMap<String, String>();

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) throws Exception {

        //   type = "BM25";
        //  type = "BM25extend_fixed";
        type = "MotiviationRandomDrifting_200";
        currentProjectName = args[0];
        //type = "BM25";

        project = "";
        topN = Integer.valueOf(args[1]);

        statusHistoryFiltering = Boolean.parseBoolean(args[2]);

         //type = args[3];
        
        int howManyTimesFirstListIsUpdated = 0;

        DataRange_Reader dataRange_Reader = new DataRange_Reader("../../../../Studied_ITS_Data/" + project
                + currentProjectName + "_issues_extra_info/" + currentProjectName + "Drift.csv");

        rangesTree = dataRange_Reader
                .getMapResults();

        SortReaderObj Sort_Reader = new SortReaderObj("../../../../Studied_ITS_Data/" + project
                + currentProjectName + "_issues_extra_info/" + currentProjectName + "Time.csv");

        sortingMap = Sort_Reader
                .getMapResults();



        if (sortingMap == null) {
            System.exit(0);
        }

        numberOf_Iterations = 100;

        if (statusHistoryFiltering) {
            ObjectInputStream ois = new ObjectInputStream(new FileInputStream(currentProjectName + "_" + topN + "NUMBERS.DATA"));
            bestSwarmPosition = (double[]) ois.readObject();
        }

        ObjectInputStream ois = new ObjectInputStream(new FileInputStream("../../../ExploratoryStudy_Classical_with_semantic/ExtendBackward/"
               + currentProjectName + "/" + currentProjectName + "_" + topN + "MotiviationrandomLimit.DATA"));

        startHashMap = (Map) ois.readObject();

        //   oos.writeObject(startHashMap);
        File folder = new File("./" + project + "" + type);
        File[] listOfFiles = folder.listFiles();

        System.out.println("Size of " + listOfFiles.length);

        Arrays.sort(listOfFiles);

        Pair[] pairs = new Pair[listOfFiles.length];
        for (int i = 0; i < listOfFiles.length; i++) {
            pairs[i] = new Pair(listOfFiles[i]);
        }

// Sort them by timestamp.
        Arrays.sort(pairs);
        for (int i = 0; i < listOfFiles.length; i++) {
            listOfFiles[i] = pairs[i].f;
        }
        for (File file : listOfFiles) {
            System.out.println(file.getName());
        }

        ExecutorService prepareThreadsPool = Executors
                .newFixedThreadPool(12);
        for (int i = listOfFiles.length - 1; i >= 0; i--) {

            DataPrepareThread dataPrepareThread = new DataPrepareThread(listOfFiles[i]);

            prepareThreadsPool.execute(dataPrepareThread);

        }

        prepareThreadsPool.shutdown();
        while (!prepareThreadsPool.isTerminated()) {
            try {
                Thread.sleep(9000L);
            } catch (Throwable throwable) {

            }
        }
        System.err.println("First wave of  Threads are Finished...");

//        prepareThreadsPool = Executors
//                .newFixedThreadPool(30);
//        for (int i = listOfFiles.length - 1; i >= 0; i--) {
//            DataPrepareThread dataPrepareThread = new DataPrepareThread(listOfFiles[i]);
//
//            prepareThreadsPool.execute(dataPrepareThread);
//
//        }
//
//        prepareThreadsPool.shutdown();
//        while (!prepareThreadsPool.isTerminated()) {
//            try {
//                Thread.sleep(9000L);
//            } catch (Throwable throwable) {
//
//            }
//        }
//        System.err.println("First wave of  Threads are Finished...");

        //  printValue(parseResultsFile("D:\\Study3\\data\\until2011\\" + project + "\\" + type + "\\" + type + "TitlesResults"), readDupsEffort("D:\\Study3\\data\\until2011\\" + project + "\\Efforts\\" + project + ".csv"));
        writeRecall("./" + project + "" + type + "_RQ2RankRecall_" + topN + "_MotiviationRealistic_statusHistoryFiltering" + statusHistoryFiltering + "_.csv");

        //startHashMap.put(names[2], values[4]);
//        writeModelRecall("./" + project + "" + type + "_FreshModelRecall_" + ".csv");
        System.out.println("The list is updated " + howManyTimesFirstListIsUpdated + " times.");

    }

    public static Object[] parseResultsFile(File filePath, Integer startId) {

        BufferedReader br = null;
        String line = "";
        String cvsSplitBy = " ";
        Boolean rangeIsCorrect = false;
        List<DupResultObj> listOfResults = new ArrayList<DupResultObj>();
        List<ResultFieldObj> resultFieldLists = null;
        long overallCount = 0;
        String[] searchToGet = filePath.getName().split("_");
        double searchSpaceSize = Double.parseDouble(searchToGet[0]) + 2;

        boolean notFoundYet = true;
        try {
            File file = filePath;
            String[] names = file.getName().split("_");

            if (file.exists()) {
                //   System.out.println("File is found successfully...");
            }
            br = new BufferedReader(new FileReader(file));

            DupResultObj dupResultObj = null;
            ResultFieldObj resultFieldObj = null;
            short count = 0;

            boolean OneIteration = false;

            while ((line = br.readLine()) != null) {

                line = line.trim().replaceAll(" +", " ");
                line = line.trim().replaceAll("= ", "=");

                //    System.out.println("~ "+line);
                String[] values = line.split(cvsSplitBy);

                //     System.out.println(line);
                if (values[0] == null || line.trim().length() == 0) {
                    continue;
                }
                if (values[0].equals("Iteration")) {

                    if (OneIteration) {
                        break;
                    }
                    OneIteration = true;
                    continue;
                }

                if (values[0].equals("Retrieving")) {

                    //      System.err.println("---> "+Integer.valueOf(values[4].trim()));
//                    if (startId > Integer.valueOf(values[4].trim())) {
//                        notFoundYet = false;
//                        continue;
//                    }

                    String startMapValue = startHashMap.get(names[2]);

                    if (!rangeIsCorrect && Integer.valueOf(values[4].trim()) >= (Integer.valueOf(startMapValue))) {
                        rangeIsCorrect = true;
                        
                        System.err.println("Range: file.getName() "+file.getName()+" ,-->,"+startMapValue+".  for "+names[2]);

                    }

                    if (!rangeIsCorrect) {
                        notFoundYet = false;
                        continue;
                    }

                    notFoundYet = true;

                    //   System.out.println("Working on Bug Iteration " + (++overallCount));
                    count = 0;
                    dupResultObj = new DupResultObj();
                    resultFieldLists = new ArrayList<ResultFieldObj>();
                    listOfResults.add(dupResultObj);
                    dupResultObj.setResultsList(resultFieldLists);

                    dupResultObj.setBug_id(values[4].trim());

                    if (sortingMap.get(Integer.valueOf(values[4].trim())) != null) {
                        dupResultObj.setDate(sortingMap.get(Integer.valueOf(values[4].trim())).getDate());
                        dupResultObj.setProductId(sortingMap.get(Integer.valueOf(values[4].trim())).getProductId());
                        dupResultObj.setComponentId(sortingMap.get(Integer.valueOf(values[4].trim())).getCompoenetId());
                        dupResultObj.setReportId(sortingMap.get(Integer.valueOf(values[4].trim())).getReportId());

                    }

                    if (values[2].equals("non-duplicate")) {
                        dupResultObj.setIsDuplicate(false);

                    } else if (values[2].equals("duplicate")) {
                        dupResultObj.setIsDuplicate(true);
                        //   System.out.println("Found duplicate");

                        dupResultObj.setMasterID(values[8].trim().substring(0, values[8].indexOf(")")));
                        //System.out.println("Master Value: "+values[8].trim());
                    }

                } else if (notFoundYet) {

                    resultFieldObj = new ResultFieldObj();
                    //   System.out.println("zz: " + values[2]);
                    //  System.out.println("zz: " + values[2] + " \nsubstring: " + values[2].substring(0, values[2].indexOf("(")));
                    resultFieldObj.setID(values[2].substring(0, values[2].indexOf("(")).trim());
              //      resultFieldObj.setSimValue(Double.valueOf(values[4]));
                    
                               if (Integer.parseInt(resultFieldObj.getID()) < 0) {
                        continue;
                    }
//                    if (count < topN) {

//                        try {
//                            if (startId > Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())) {
//                                //  System.out.println("values[2].substring(0, values[2].indexOf(\"(\")).trim()= "+values[2].substring(0, values[2].indexOf("(")).trim());
//                                continue;
//                            }
//                        } catch (java.lang.ArrayIndexOutOfBoundsException boundsExceptio) {
//
//                            /// System.out.println(line);
//                            //System.out.println(boundsExceptio);
//                            boundsExceptio.printStackTrace();
//                            System.exit(0);
//                        }
                    if (sortingMap != null) {
                        if (sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())) != null) {
                            resultFieldObj.setProductId(sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())).getProductId());
                            resultFieldObj.setComponentId(sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())).getCompoenetId());
                            resultFieldObj.setReportId(sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())).getReportId());

                        }

                        if (sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())) != null) {
                            resultFieldObj.setDate(sortingMap.get(Integer.valueOf(values[2].substring(0, values[2].indexOf("(")).trim())).getDate());

                            if (dupResultObj.getDate() != null) {
                                resultFieldObj.setDiffTime((double) (dupResultObj.getDate().getTime() - resultFieldObj.getDate().getTime()) / (double) (1000 * 60 * 60 * 24));
                            }
                        } else {
                            System.out.println("Didn't find  bug ID date for: " + values[2].substring(0, values[2].indexOf("(")).trim());
                        }

                        if (sortingMap.get(Integer.valueOf(resultFieldObj.getID())) != null && sortingMap.get(Integer.valueOf(resultFieldObj.getID())).getResolutionsDate() != null) {
                            resultFieldObj.setResolutionDate(sortingMap.get(Integer.valueOf(resultFieldObj.getID())).getResolutionsDate());
                        } else {
//                        System.out.println("Didn't find  resolution  bug ID date for: " + resultFieldObj.getID());

                        }

                        if (sortingMap.get(Integer.valueOf(resultFieldObj.getID())) != null && sortingMap.get(Integer.valueOf(resultFieldObj.getID())).getAnyResoDate() != null) {
                            resultFieldObj.setResolutionAll(sortingMap.get(Integer.valueOf(resultFieldObj.getID())).getAnyResoDate());
                        } else {
//                        System.out.println("Didn't find  resolution  bug ID date for: " + resultFieldObj.getID());

                        }

                    }

                    if (statusHistoryFiltering && dupResultObj.isIsDuplicate()) {
                        if (count >= 0) {
//                        if(Integer.valueOf(dupResultObj.getBug_id())-Integer.valueOf(resultFieldObj.getID())>2000)
//                        {
//                            continue;
//                        }

                            SortObj sortObj = sortingMap.get(Integer.valueOf(resultFieldObj.getID()));

//                            System.out.println("Checking for the Query - >" + dupResultObj.getBug_id());
                            System.out.println("--- + Checking for the Results - >" + resultFieldObj.getID());
                            if (sortObj != null && dupResultObj.getDate() != null) {
                                if (!sortObj.checkIfStillOpen(dupResultObj.getDate())) {
//                                    System.out.println("--- + Found data for Checking for the Results - >" + resultFieldObj.getID());
                                    Date matchDate = sortObj.getCloseDateForThis(dupResultObj.getDate());
//                                    System.out.println("--- + Query Date - >" + dupResultObj.getDate() + " " + "--- + Target Date - >" + matchDate);

                                    if (matchDate != null) {

                                        long startTime = dupResultObj.getDate().getTime();
                                        long endTime = matchDate.getTime();
                                        long diffTime = startTime - endTime;
                                        long diffDays = diffTime / (1000 * 60 * 60 * 24);
                                        System.out.println("--- + Diff in days - >" + diffDays);
                                        double val = sortObj.returnTimeThreshold(bestSwarmPosition);
                                        System.out.println("Thrshold value= " + val);

                                        countResolvedAtTime++;
                                        double limit = 0;
                                        if (searchSpaceSize > 0) {
                                            limit = ((74 / Math.exp(count)) * val);
                                        } else {
                                            limit = 90;
                                        }
//                                        double limit = ((searchSpaceSize / ((double) ((count + 1) * (count + 1)))) * val);
//                                        if (count == 0 && limit < 60) {
//                                            limit = 60;
//                                        }
                                        if (diffDays > limit) {
                                            if (dupResultObj.getMasterID().equals(resultFieldObj.getID())) {
//                                                System.out.println(diffDays + "  Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
//                                                System.out.println(diffDays + "  Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
                                            }
                                            continue;

                                        } else if (dupResultObj.getMasterID().equals(resultFieldObj.getID())) {
//                                            System.out.println(diffDays + " larger Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
//                                            System.out.println(diffDays + "larger  Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
                                        }

                                    } else {
                                        counMatchMissedOpen++;
//                                        System.out.println(resultFieldObj.getDiffTime() + "openopen larger Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
//                                        System.out.println(resultFieldObj.getDiffTime() + "openopen  larger  Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());

                                    }
                                } else {

//                                    if (dupResultObj.isIsDuplicate()) {
//                                        if (count >= 0) {
//
//                                            if (dupResultObj.getReportId().equals(resultFieldObj.getReportId())) {
//
//                                                System.out.println("Fuck am here");
//                                                continue;
//                                            }
//                                        }
//                                    }
                                    countOpenResolvedAtTime++;
                                    //System.out.println("It is open");
//                                    System.out.println(resultFieldObj.getDiffTime() + " -- Count =  " + count + " openopen larger Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
//                                    System.out.println(resultFieldObj.getDiffTime() + " -- Count =  " + count + "openopen  larger  Query--> " + dupResultObj.getBug_id() + "--- + Filter out  ID - >" + resultFieldObj.getID());
                                }

                            } else {
                                countMissedResolvedAtTime++;
                            }

//                        if (resultFieldObj.getResolveCase() == 0) {
////                             Ignored resolved...
//                            continue;
//                        }
                        }
                    }
                    if(Integer.parseInt(resultFieldObj.getID())<0)
                    {
                        continue;
                    }
                    

                    resultFieldObj.setRank(++count);
                    resultFieldLists.add(resultFieldObj);
                    if (topN == count) {
                        notFoundYet = false;
                    }
                }
//                }

                //   System.gc();
            }

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        Object[] ObjectArr = new Object[2];
        ObjectArr[0] = listOfResults;
        ObjectArr[1] = rangeIsCorrect;

        //Object[] objectList=Object[]{listOfResults};
        return ObjectArr;
    }

    public static void printValue(List<DupResultObj> dupResultObjs, Map<String, EffortRecordObj> mapEffort, String key,
                                  Integer trainingSize, int StartId, int month, int year, int distance,
                                  List<DupResultObj> generalCompareList,
                                  List<DupResultObj> foundList,
                                  List<DupResultObj> notFoundList,
                                  List<DupResultObj> missedEffortRecord) throws Exception {

        int countNotDups = 0;

        int foundMaster = 0;
        double effortFound = 0;
        double effortNoFound = 0;
        double effortMatch = 0;
        double countDups = 0;
        countDups = 0;
        double effortCountFound = 0;
        double effortCountNotFound = 0;

        double indexMAP = 0.0d;

        int effortMissedRecord = 0;

        List<Double> foundMedian = new ArrayList();
        List<Double> notFoundMedian = new ArrayList();
        generalCompareList = dupResultObjs;

        if (dupResultObjs.size() != generalCompareList.size()) {
            throw new Exception("Error:  dupResultObjs.size()!=generalCompareList.size() ");
        }

        int dupCount = 0;
        int dupNotCount = 0;
        for (DupResultObj dro : generalCompareList) {

            if (dro.isIsDuplicate()) {
                dupCount++;
            } else {
                dupNotCount++;
            }
        }
        System.out.println("dupCount ============================ " + dupCount + "   ---> Not: " + dupNotCount);

        for (int k = 0; k < dupResultObjs.size(); k++) {

            DupResultObj dro = dupResultObjs.get(k);
            // System.out.println("Query Bug ID: " + dro.getBug_id() + " MasterID=" + dro.getMasterID());
            //generalCompareList

            if (generalCompareList.get(k).isIsDuplicate() && generalCompareList.get(k).getBug_id().equals(dro.getBug_id())) {
                countDups++;
            } else {

                if (dro.isIsDuplicate() && !generalCompareList.get(k).isIsDuplicate()) {
                    throw new Exception("Yes it happens ");
                }

                countNotDups++;
            }

            int countOfList = 0;
            boolean flagFound = false;
            for (ResultFieldObj fieldObj : dro.getResultsList()) {

                // System.out.println("Rank-- " + fieldObj.getRank() + " , ID: " + fieldObj.getID());
                if (generalCompareList.get(k).isIsDuplicate()) {

//                    if (Integer.valueOf(fieldObj.getID()) < StartId) {
//                        continue;
//                    }
                    if (generalCompareList.get(k).getMasterID().equals(fieldObj.getID())) {
                        //  System.out.println("Found Master:" + dro.getMasterID() + " , for :" + fieldObj.getID());
                        foundMaster++;

                        flagFound = true;
                        dro.setRank(fieldObj.getRank());
                        dro.setSimilarityToMaster(fieldObj.getSimValue());

                        if (generalCompareList.get(k).getSimilarityToMaster() > maxSimilarity) {
                            maxSimilarity = generalCompareList.get(k).getSimilarityToMaster();
                        }

                        for (int j = topN - 1; j >= 0; j--) {
                            counCorrectRank[j]++;
                            if (j == countOfList - 1) {
                                break;
                            }
                        }

                    }

                    if (flagFound) {
                        indexMAP += 1.0d / ((double) countOfList + 1);
                        break;
                    }

                }

                countOfList++;

                if (countOfList == topN) {
                    break;
                }

            }

            if (dro.isIsDuplicate() && generalCompareList.get(k).isIsDuplicate() && generalCompareList.get(k).getBug_id().equals(dro.getBug_id())) {
                if (mapEffort.get(dro.getBug_id()) != null) {
                    effortMatch++;
                    if (maxDupsExp < mapEffort.get(dro.getBug_id()).getExperianceDups()) {
                        maxDupsExp = mapEffort.get(dro.getBug_id()).getExperianceDups();
                    }

                    if (flagFound) {
                        effortFound += mapEffort.get(dro.getBug_id()).getTimeDelay();
                        effortCountFound++;
                        dro.setDupEffortObj(mapEffort.get(dro.getBug_id()));

                        foundList.add(dro);

                        foundMedian.add(mapEffort.get(dro.getBug_id()).getTimeDelay());

                    } else {
                        effortNoFound += mapEffort.get(dro.getBug_id()).getTimeDelay();
                        effortCountNotFound++;
                        dro.setDupEffortObj(mapEffort.get(dro.getBug_id()));
                        notFoundList.add(dro);

                        notFoundMedian.add(mapEffort.get(dro.getBug_id()).getTimeDelay());

                    }
                } else {

                    effortMissedRecord++;
                    missedEffortRecord.add(dro);
                    //  System.out.println("--Effort Match Missed... " + dro.getBug_id());
                }

            }

        }

        System.out.println("Dups Count: " + countDups);
        System.out.println("No-Dups Count: " + countNotDups);
        System.out.println("Found Master: " + foundMaster);
        System.out.println("Recall: " + (double) foundMaster / (double) countDups);

        System.out.println("------------------------------------------------");

//        System.out.println("Matched Effort Found: " + effortCountFound);
//        System.out.println("Matched Effort Not-Found: " + effortCountNotFound);
//        System.out.println("Missed Records to Effort: " + effortMissedRecord);
//        System.out.println("Effort Found: " + effortFound / effortCountFound);
//        System.out.println("Effort Not-Found: " + effortNoFound / effortCountNotFound);
        System.out.println("key :" + key);

        TreeMap<Integer, DriftObj> tm = null;
        if (mapResults.get(key) == null) {
            tm = new TreeMap<Integer, DriftObj>();
        } else {
            tm = mapResults.get(key);
        }
        DriftObj objDrift = new DriftObj();
        objDrift.setRecall((double) foundMaster / (double) countDups);
        objDrift.setNumbOfDups((long) countDups);

        String[] values = key.split("_");
        
        objDrift.setMonth(month);
        objDrift.setYear(year);
        objDrift.setTrainingSize(trainingSize);
        objDrift.setDistance(distance);
        objDrift.setNotDups(foundMaster);
//        objDrift.setStartTrainingID(values[5]);
//        objDrift.setEndTrainingID(values[6]);
//        objDrift.setRatioOfTraining(values[7]);
        
        

//        objDrift.setNotFoundMedian(findMedian(foundMedian));
//        objDrift.setFoundMedian(findMedian(notFoundMedian));
        objDrift.setMAP((double) indexMAP / (double) foundMaster);

        tm.put(trainingSize, objDrift);
        mapResults.put(key, tm);

        if (distance == 0) {
            tm = new TreeMap<Integer, DriftObj>();
            tm.put(trainingSize, objDrift);
            mapModels.put(key, tm);
        }

        try {
//            writeDupObjToCSV(foundList, "D:\\Study3\\DriftResults\\" + project + "\\Found\\" + type + "_" + key + "_Found_" + ".csv");
//            writeDupObjToCSV(notFoundList, "D:\\Study3\\DriftResults\\" + project + "\\Notfound\\" + type + "_NotFound_" + ".csv");
//            writeDupObjToCSV(missedEffortRecord, "D:\\Study3\\DriftResults\\" + project + "\\" + type + "_EffortMissedRecord_" + ".csv");
        } catch (Throwable throwable) {
            throwable.printStackTrace();
        }

    }

    public static double findMedian(List<Double> listArr) {

        Collections.sort(listArr);
        double median;
        if (listArr.size() % 2 == 0) {
            median = ((double) listArr.get(listArr.size() / 2) + (double) listArr.get(listArr.size() / 2 - 1)) / 2;
        } else {
            median = (double) listArr.get(listArr.size() / 2);
        }
        return median;
    }

    public static Map<String, EffortRecordObj> readDupsEffort(String filePath) {

        BufferedReader br = null;
        String line = "";
        String cvsSplitBy = ",";
        Map<String, EffortRecordObj> effortObjsList = new HashMap<String, EffortRecordObj>();
        EffortRecordObj deo = null;

        try {

            br = new BufferedReader(new FileReader(filePath));

            boolean firstTime = true;

            while ((line = br.readLine()) != null) {

                String[] values = line.split(cvsSplitBy);

                if (!firstTime) {
                    deo = new EffortRecordObj();

                    deo.setBug_id(values[0]);
                    deo.setMasterId(values[1]);
                    deo.setTimeDelay(Double.valueOf(values[2]));

                    if (maxTimeDelay < Double.valueOf(values[2])) {
                        maxTimeDelay = Double.valueOf(values[2]);
                    }

                    deo.setPeopleInvolved(Double.valueOf(values[4]));

                    if (maxPPLDelay < Double.valueOf(values[4])) {
                        maxPPLDelay = Double.valueOf(values[4]);
                    }

                    deo.setCommentsEffort(Double.valueOf(values[3]));

                    deo.setCorrectedTimeDelay(Double.valueOf(values[5]));
                    deo.setWhoIdentified(values[6]);
                    deo.setExperianceDups(Double.valueOf(values[7]));

//                    if (maxDupsExp < Double.valueOf(values[7])) {
//                        maxDupsExp = Double.valueOf(values[7]);
//                    }
                    if (maxCommentsDelay < Double.valueOf(values[3])) {
                        maxCommentsDelay = Double.valueOf(values[3]);
                    }

                    if (values[8].length() > 0) {
                        deo.setDescriptionSim(Double.valueOf(values[8]));
                    }
                    deo.setPeersCCList(Integer.valueOf(values[9]));
                    deo.setClosetFixedIssue(Double.valueOf(values[10]));
                    deo.setClosetCreatedPeer(Double.valueOf(values[11]));
                    deo.setClosetDetectedPeer(Double.valueOf(values[12]));
                    deo.setDayHour(Integer.valueOf(values[13]));
                    deo.setYearDay(Integer.valueOf(values[14]));
                    deo.setWaitingLoad(Integer.valueOf(values[15]));
                    deo.setComponent(values[16]);
                    deo.setDescriptionSize(Integer.valueOf(values[17]));
                    deo.setTitleSize(Integer.valueOf(values[18]));
                    deo.setPriority(Integer.valueOf(values[19]));
                    deo.setSeverity(Integer.valueOf(values[20]));

                    //eo.s
                    effortObjsList.put(values[0], deo);
                }

                firstTime = false;
            }

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return effortObjsList;
    }

    public static void writeDupObjToCSV(List<DupResultObj> dupResultObjsListToWrite, String filePath) {

        try {
            FileWriter writer = new FileWriter(filePath, false);

            writer.append("bug_id");
            writer.append(',');

            writer.append("masterID");
            writer.append(',');
            writer.append("isDuplicate");
            writer.append(',');
            writer.append("timeDelay");
            writer.append(',');
            writer.append("CorrectedTimeDelay");
            writer.append(',');

            writer.append("whoIdentified");
            writer.append(',');

            writer.append("expDups");
            writer.append(',');

            writer.append("commentsEffort");
            writer.append(',');
            writer.append("peopleInvolved");
            writer.append(',');
            writer.append("EffortFactor");

            writer.append(',');
            writer.append("NormEffFactor");
            writer.append(',');
            writer.append("NormDupExp");

            writer.append(',');
            writer.append("Rank");

            writer.append(',');
            writer.append("SimilarityToMaster");
            writer.append(',');
            writer.append("normSimToMaster");

            writer.append(',');
            writer.append("Description Similarity");
            writer.append(',');
            writer.append("Peers CC list");
            writer.append(',');
            writer.append("Closest Fixed Issue");
            writer.append(',');
            writer.append("Closest Created Peer");
            writer.append(',');
            writer.append("Closest Detected Peer");
            writer.append(',');
            writer.append("Day Hour");
            writer.append(',');
            writer.append("Year Day");
            writer.append(',');
            writer.append("Waiting Load");
            writer.append(',');
            writer.append("Component");
            writer.append(',');
            writer.append("Description Size");

            writer.append(',');
            writer.append("Title Size");
            writer.append(',');
            writer.append("Priority");

            writer.append(',');
            writer.append("Severity");

            writer.append('\n');

            for (DupResultObj dupResultObj : dupResultObjsListToWrite) {

                writer.append(dupResultObj.getBug_id());
                writer.append(',');

                writer.append(dupResultObj.getMasterID());
                writer.append(',');
                writer.append(Boolean.toString(dupResultObj.isIsDuplicate()));
                writer.append(',');

                EffortRecordObj dupEffortObj = dupResultObj.getDupEffortObj();
                if (dupEffortObj != null) {
                    writer.append(Double.toString(dupEffortObj.getTimeDelay()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getCorrectedTimeDelay()));
                    writer.append(',');

                    writer.append(dupEffortObj.getWhoIdentified());
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getExperianceDups()));
                    writer.append(',');

                    writer.append(Double.toString(dupEffortObj.getCommentsEffort()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getPeopleInvolved()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getTimeDelay() * (1 + dupEffortObj.getCommentsEffort())));
                    writer.append(',');
                    writer.append(Double.toString((dupEffortObj.getTimeDelay() / maxTimeDelay) + (dupEffortObj.getCommentsEffort() / maxCommentsDelay) + (dupEffortObj.getPeopleInvolved() / maxPPLDelay)));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getExperianceDups() / maxDupsExp));
                    writer.append(',');
                    writer.append(Double.toString(dupResultObj.getRank()));
                    writer.append(',');
                    writer.append(Double.toString(dupResultObj.getSimilarityToMaster()));
                    writer.append(',');
                    writer.append(Double.toString((double) dupResultObj.getSimilarityToMaster() / (double) maxSimilarity));

                    /////
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getDescriptionSim()));
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getPeersCCList()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getClosetFixedIssue()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getClosetCreatedPeer()));
                    writer.append(',');
                    writer.append(Double.toString(dupEffortObj.getClosetDetectedPeer()));
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getDayHour()));
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getYearDay()));
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getWaitingLoad()));
                    writer.append(',');
                    writer.append(dupEffortObj.getComponent());
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getDescriptionSize()));

                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getTitleSize()));
                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getPriority()));

                    writer.append(',');
                    writer.append(Integer.toString(dupEffortObj.getSeverity()));

                } else {
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");

                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");

                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");
                    writer.append(',');
                    writer.append("");

                }

//                writer.append(',');
//                writer.append(Short.toString(dupResultObj.getRank()));
//                writer.append(',');
//                writer.append(Double.toString(dupResultObj.getSimilarityToMaster()));
//                writer.append(',');
//                writer.append(Double.toString((double) dupResultObj.getSimilarityToMaster() / (double) maxSimilarity));
//                writer.append(',');
//                writer.append(Double.toString(dupEffortObj.getDescriptionSim()));
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getPeersCCList()));
//                writer.append(',');
//                writer.append(Double.toString(dupEffortObj.getClosetFixedIssue()));
//                writer.append(',');
//                writer.append(Double.toString(dupEffortObj.getClosetCreatedPeer()));
//                writer.append(',');
//                writer.append(Double.toString(dupEffortObj.getClosetDetectedPeer()));
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getDayHour()));
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getYearDay()));
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getWaitingLoad()));
//                writer.append(',');
//                writer.append(dupEffortObj.getComponent());
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getDescriptionSize()));
//
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getTitleSize()));
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getPriority()));
//
//                writer.append(',');
//                writer.append(Integer.toString(dupEffortObj.getSeverity()));
                writer.append('\n');

            }

            writer.flush();
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static void writeRecall(String filePath) {

        try {
            FileWriter writer = new FileWriter(filePath, false);
            writer.append("Data_Range");
            writer.append(',');
            writer.append("trainingSize");
            writer.append(',');
            writer.append("recall");
            writer.append(',');
            writer.append("lastpoint");
            writer.append(',');
            writer.append("countDups");

            writer.append(',');
            writer.append("month");
            writer.append(',');
            writer.append("month_name");
            writer.append(',');
            writer.append("year");
            writer.append(',');
            writer.append("distance");
            writer.append(',');
            writer.append("trainingSize");

            writer.append(',');
            writer.append("FoundMaster");

            writer.append(',');
            writer.append("MAP");

            writer.append(',');
            writer.append("startTrainingID");

            writer.append(',');
            writer.append("endTrainingID");
            writer.append(',');
            writer.append("RatioOfSize");

            
            writer.append('\n');
            DriftObj preRecallObj = null;
            for (String tmStr : mapResults.keySet()) {
                TreeMap<Integer, DriftObj> tm = mapResults.get(tmStr);

                for (Integer keyInt : tm.keySet()) {

                    DriftObj recallObj = tm.get(keyInt);
                    writer.append(tmStr);
                    writer.append(',');
                    writer.append(Integer.toString(keyInt));
                    writer.append(',');
                    writer.append(Double.toString(recallObj.getRecall()));
                    writer.append(',');

                    preRecallObj = getPreviousDistance(recallObj.getDistance(), recallObj);
                    String[] keys = tmStr.split("_");

                    writer.append("<-" + keys[0]);
                    writer.append(',');
                    writer.append(Long.toString(recallObj.getNumbOfDups()));

                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getMonth()));
                    writer.append(',');
                    Date d = new Date(recallObj.getYear(), recallObj.getMonth() - 1, 1);

                    //writer.append(new SimpleDateFormat("mm/yyyy").format(d));
                    writer.append(Integer.toString(recallObj.getMonth()) + "/" + Integer.toString(recallObj.getYear()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getYear()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getDistance() + 2));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getTrainingSize()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getNotDups()));

                    writer.append(',');
                    writer.append(Double.toString(recallObj.getMAP()));

                    writer.append(',');
                    writer.append(recallObj.getStartTrainingID());
                    writer.append(',');
                    writer.append(recallObj.getEndTrainingID());
                    writer.append(',');
                    writer.append(recallObj.getRatioOfTraining());

                    

                    writer.append('\n');

                }

            }

            writer.flush();
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static DriftObj getPreviousDistance(int distance, DriftObj org) {
        for (String tmStr : mapResults.keySet()) {
            TreeMap<Integer, DriftObj> tm = mapResults.get(tmStr);
            for (Integer keyInt : tm.keySet()) {

                DriftObj recallObj = tm.get(keyInt);

                if (recallObj.getDistance() == (distance - 2) && org.getMonth() == recallObj.getMonth() && org.getYear() == recallObj.getYear()) {

                    return recallObj;
                }

            }
        }

        return org;

    }

    public static void writeModelRecall(String filePath) {

        try {
            FileWriter writer = new FileWriter(filePath, false);
//            writer.append("Data_Range");
//            writer.append(',');
            writer.append("trainingSize");
            writer.append(',');
            writer.append("recall");
            writer.append(',');
            writer.append("countDups");

            writer.append(',');
            writer.append("month");
            writer.append(',');
//            writer.append("month_name");
//            writer.append(',');
            writer.append("year");
            writer.append(',');
            writer.append("distance");
            writer.append(',');
            writer.append("trainingSize");

            writer.append(',');
            writer.append("FoundMaster");

            writer.append(',');
            writer.append("MAP");

            writer.append(',');
            writer.append("foundMedian");

            writer.append(',');
            writer.append("notFoundMedian");
            writer.append(',');
            writer.append("normComponentCount");
            writer.append(',');
            writer.append("normAuthorCount");
            writer.append(',');
            writer.append("normVersionCount");

            writer.append('\n');
            for (String tmStr : mapModels.keySet()) {
                TreeMap<Integer, DriftObj> tm = mapModels.get(tmStr);

                for (Integer keyInt : tm.keySet()) {

                    DriftObj recallObj = tm.get(keyInt);
//                    writer.append(tmStr);
//                    writer.append(',');
                    writer.append(Integer.toString(keyInt));
                    writer.append(',');
                    writer.append(Double.toString(recallObj.getRecall()));
                    writer.append(',');
                    writer.append(Long.toString(recallObj.getNumbOfDups()));

                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getMonth()));
//                    writer.append(',');
//                    Date d = new Date(recallObj.getYear(), recallObj.getMonth() - 1, 1);
//
//                    //writer.append(new SimpleDateFormat("mm/yyyy").format(d));
//                    writer.append(Integer.toString(recallObj.getMonth()) + "/" + Integer.toString(recallObj.getYear()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getYear()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getDistance() + 2));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getTrainingSize()));
                    writer.append(',');
                    writer.append(Integer.toString(recallObj.getNotDups()));

                    writer.append(',');
                    writer.append(Double.toString(recallObj.getMAP()));

                    writer.append(',');
                    writer.append(Double.toString(recallObj.getFoundMedian()));
                    writer.append(',');
                    writer.append(Double.toString(recallObj.getNotFoundMedian()));

                    TreeMap<Integer, DataRange> treeMapMonths = rangesTree
                            .get(recallObj.getYear());
                    DataRange dataRange = treeMapMonths.get(recallObj.getMonth());

                    writer.append(',');
                    writer.append(Double.toString(dataRange.getNumOfComponent() / (double) dataRange.getNumberOfReports()));
                    writer.append(',');
                    writer.append(Double.toString(dataRange.getNumOfAuthors() / (double) dataRange.getNumberOfReports()));
                    writer.append(',');
                    writer.append(Double.toString(dataRange.getNumOfVersionCount() / (double) dataRange.getNumberOfReports()));

                    writer.append('\n');
                }

            }

            writer.flush();
            writer.close();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }
}

class Pair implements Comparable {

    public long t;
    public File f;

    public Pair(File file) {
        f = file;
        String[] values = file.getName().split("_");
        t = Long.parseLong(values[0]);
    }

    public int compareTo(Object o) {
        long u = ((Pair) o).t;
        return t < u ? -1 : t == u ? 0 : 1;
    }
};

class DataPrepareThread extends Thread {

    File file;
    List<DupResultObj> generalCompareList;
    public List<DupResultObj> foundList;
    public List<DupResultObj> notFoundList;
    public List<DupResultObj> missedEffortRecord;
    public boolean rangeIsCorrect;

    public DataPrepareThread(File newFile) {
        this.file = newFile;
    }

    @Override
    public void run() {
        try {
            // TODO Auto-generated method stub
            executeCommand();
            System.err.println("***** End thread ***** \n " + file);
        } catch (Exception ex) {
            Logger.getLogger(DataPrepareThread.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public void executeCommand() throws Exception {

        System.err.println("***** Starting thread ***** \n " + file);

        foundList = new ArrayList<DupResultObj>();
        notFoundList = new ArrayList<DupResultObj>();
        missedEffortRecord = new ArrayList<DupResultObj>();
        rangeIsCorrect = false;
        maxTimeDelay = 0;
        maxCommentsDelay = 0;
        maxPPLDelay = 0;
        maxDupsExp = 0;
        maxFixExp = 0;
        maxSimilarity = 0;
        counCorrectRank = new double[topN];

        String nameStr = file.getName();
        String[] values = nameStr.split("_");
        //values[1]
        //values[2]
        //values[3]
        System.err.println("listOfFiles[i]-->  " + file);
        Object[] arrObjs = parseResultsFile(file, Integer.valueOf(values[1]));

        List<DupResultObj> dupResultObjs = (List<DupResultObj>) arrObjs[0];
        rangeIsCorrect = (Boolean) arrObjs[1];
        if (!rangeIsCorrect) {
            System.err.println("Exception---------------> Wrong range for " + nameStr + " : startID: " + Integer.valueOf(values[1]));

            System.exit(0);
        }

        //    if (i == 0) {
        if (treeMapDupResults.get(values[4] + "_" + values[5]) == null) {
            if (Integer.valueOf(values[0]).equals(Drift_Fresh_FreshExtended.numberOf_Iterations - 2)) {
                generalCompareList = dupResultObjs;
                treeMapDupResults.put(values[4] + "_" + values[5], generalCompareList);

                //howManyTimesFirstListIsUpdated++;
                System.err.println(" Testing ..... creating new one " + values[4] + "_" + values[5]);
            } else {
                System.err.println("Error:  Something is wrong");
            }
        } else {

            generalCompareList = treeMapDupResults.get(values[4] + "_" + values[5]);

            System.err.println(" Testing ..... Found Old List for " + values[4] + "_" + values[5] + "  -->" + generalCompareList.size() + " new" + dupResultObjs.size());
        }

        if (!Integer.valueOf(values[0]).equals(Drift_Fresh_FreshExtended.numberOf_Iterations - 2) && dupResultObjs.equals(generalCompareList)) {
            System.err.println("Error:  dupResultObjs.size()==generalCompareList.size()  they are clonnnes");
        }

        //    }
        //readDupsEffort("D:\\Study3\\data\\drift\\" + project + "\\Efforts\\" + project + ".csv");
        Map<String, EffortRecordObj> mapEffort = readDupsEffort("../../../../Studied_ITS_Data/" + project
                + currentProjectName + "_issues_extra_info/" + currentProjectName + ".csv");

        System.err.println("Dup Results Objs-->  " + dupResultObjs.size());
        printValue(dupResultObjs, mapEffort,
                values[5] + "_" + values[1] + "_" + values[2] + "_" + values[4] + "_" + values[0],
                Integer.valueOf(values[3]),
                Integer.valueOf(values[1]),
                Integer.valueOf(values[4]),
                Integer.valueOf(values[5]),
                Integer.valueOf(values[0]), generalCompareList, foundList, notFoundList, missedEffortRecord);

    }

}
